{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "import re\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import SnowballStemmer\n",
    "import pymongo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                               Sentence  polit\n",
      "2921  This morning reported guests asked stop remain...      1\n",
      "8673  El Camino: A Breaking Bad Movie actor Johnny O...      2\n",
      "8361  Read Time: This article based research finding...      2\n",
      "6532  Infinity Ward renamed controversial soldier sk...      2\n",
      "4174  Kanye West's  presidential campaign still unde...      1\n",
      "...                                                 ...    ...\n",
      "4963  The American Academy Pediatrics Friday flipped...      2\n",
      "5645  On July , , Netflix thrilled mystery craving f...      2\n",
      "1749  The Atlanta Wendy’s Rayshard Brooks killed pol...      1\n",
      "7397  Palm Springs, new movie Hulu, stars Andy Sambe...      2\n",
      "23    Historical human activity We used publications...      1\n",
      "\n",
      "[9497 rows x 2 columns]\n",
      "{0:                                                Sentence  polit\n",
      "5837  Director Center Health Security Johns Hopkins ...      2\n",
      "2524  Every  months, orbits Earth Mars conspire make...      1\n",
      "3597  Photos Facebook CEO Mark Zuckerberg surfing Ha...      1\n",
      "5288  (CNN) House Speaker Nancy Pelosi Saturday slam...      2\n",
      "167   “Having schools come back traditional model, s...      1\n",
      "...                                                 ...    ...\n",
      "217   Britney Spears reacting \"Toxic\" comments Insta...      1\n",
      "8639  Senior Democrats House Representatives oversig...      2\n",
      "3738  Getty \"You definitely believe everything read....      1\n",
      "5859  Surgeon General Jerome Adams said Sunday US co...      2\n",
      "1218  The first over-the-air update hit Galaxy S Gal...      1\n",
      "\n",
      "[380 rows x 2 columns], 1:                                                Sentence  polit\n",
      "715   © @CostcoBuys - Instagram Costco sells several...      1\n",
      "1099  Google's newest ad ban classic \"better late ne...      1\n",
      "1691  Microsoft added crafty Bing Search shortcut An...      1\n",
      "8530  What happened Shares precious metals miner Ame...      2\n",
      "9300  Feeling sick fever — even slight one — along c...      2\n",
      "...                                                 ...    ...\n",
      "5044  Credit: Dr Dimitar Epihov Adding crushed rock ...      2\n",
      "479   A comet-eating black hole size planet? It’s po...      1\n",
      "1542  Louisiana's Attorney General Jeff Landry (R) a...      1\n",
      "916   ESPN suspends Adrian Wojnarowski, top NBA repo...      1\n",
      "5713  The White House seeking discredit Dr. Anthony ...      2\n",
      "\n",
      "[380 rows x 2 columns], 2:                                                Sentence  polit\n",
      "2599  Image copyright Getty Images Image caption Min...      1\n",
      "1698  Photo : Emma McIntyre Entertainment Weekly ( G...      1\n",
      "6874  It's new week, means Call Duty: Modern Warfare...      2\n",
      "2672  Lenovo release high-spec gaming phone rarely (...      1\n",
      "878   Mr. Trump wore dark mask affixed presidential ...      1\n",
      "...                                                 ...    ...\n",
      "5952  (CNN) One three young adults risk severe Covid...      2\n",
      "1005  New York (CNN Business) A coalition Black advo...      1\n",
      "4305  Image copyright Getty Images Authorities Germa...      1\n",
      "147   (CNN) Facebook removed dozens said fake social...      1\n",
      "4548  (CNN) White House coronavirus task force coord...      1\n",
      "\n",
      "[380 rows x 2 columns], 3:                                                Sentence  polit\n",
      "5324  Advertisement The governor Texas warned could ...      2\n",
      "9060  Priscilla Presley speaking losing grandson, Be...      2\n",
      "7131  WASHINGTON—Senior Trump administration officia...      2\n",
      "2558  An abandoned decaying oil tanker Red Sea could...      1\n",
      "1647  The , reservation spots first run revived Ford...      1\n",
      "...                                                 ...    ...\n",
      "4941  Face-masked citizens lined general elections S...      2\n",
      "3307  If craving chicken nuggets, perhaps I interest...      1\n",
      "7564  California, Florida Texas accounted half new U...      2\n",
      "733   Warriors star Steph Curry simply visiting Howa...      1\n",
      "1104  Samsung Electronics recently named beloved bra...      1\n",
      "\n",
      "[380 rows x 2 columns], 4:                                                Sentence  polit\n",
      "5009  When MLB announced would -game season, project...      2\n",
      "5766  Tucker Carlson's top writer Blake Neff resigne...      2\n",
      "7394  The latest service join streaming wars none NB...      2\n",
      "3220  President Trump blasted niece’s tell-all book ...      1\n",
      "6124  Just A Kid According sophisticated new model, ...      2\n",
      "...                                                 ...    ...\n",
      "1861  Xiaomi took stage today online-only event show...      1\n",
      "5080  Several months Jordyn Woods ousted Kardashian-...      2\n",
      "1751  Subtlety exactly  Mercedes-AMG GT Black Series...      1\n",
      "1574  (CNN) There little doubt former Vice President...      1\n",
      "7252  New Orleans Pelicans’ Zion Williamson left NBA...      2\n",
      "\n",
      "[380 rows x 2 columns], 5:                                                Sentence  polit\n",
      "3277  Minecraft fans finally discovered world seed M...      1\n",
      "5789  Suda cheeky bastard A weeks ago New Game+ Expo...      2\n",
      "5452  The promising-looking Carrion comes rd July PC...      2\n",
      "8454  PROVO, Utah — Whether wear mask coronavirus pa...      2\n",
      "2665  We’ve seen lot Samsung’s Note  Ultra date, hav...      1\n",
      "...                                                 ...    ...\n",
      "3817  Supernovae powerful events Universe. They’re e...      1\n",
      "1788  Elon Musk posted first official render appears...      1\n",
      "5271  Remember super rare sticker sealed copy Super ...      2\n",
      "5562  RAPID CITY, Mich. – The Torch Lake sandbar lis...      2\n",
      "5873  'I hide bathroom cry': Americans struggle unem...      2\n",
      "\n",
      "[380 rows x 2 columns], 6:                                                Sentence  polit\n",
      "2297  Enlarge Image Ford If hype overwhelmingly posi...      1\n",
      "4801  For past several weeks, OnePlus teasing upcomi...      2\n",
      "3508  Are Garcelle Beauvais Lisa Rinna longer friend...      1\n",
      "7037  Thousands companies face restrictions storing ...      2\n",
      "1234  More dozen tech companies filed brief Monday f...      1\n",
      "...                                                 ...    ...\n",
      "6183  President Trump Monday sought redirect blame c...      2\n",
      "6327  Due current pandemic there’s way knowing  NFL ...      2\n",
      "8736  A report calls question providers’ security pr...      2\n",
      "5707  Coronavirus infectious disease confirmed  mill...      2\n",
      "7050  ATHENS — Greg Sankey said SEC still considerin...      2\n",
      "\n",
      "[380 rows x 2 columns], 7:                                                Sentence  polit\n",
      "351   Online retailer Shein apologized one product l...      1\n",
      "483   The United States Army heart like college frat...      1\n",
      "2421  We're fans smart displays WIRED Gear team. The...      1\n",
      "388   Amy Schumer always keeps real. So, come surpri...      1\n",
      "9104  Chris Evans: actor, superhero, — recently — po...      2\n",
      "...                                                 ...    ...\n",
      "6796  Get breaking news alerts special reports. The ...      2\n",
      "6892  (CNN) A statue British imperialist Cecil Rhode...      2\n",
      "2164  Nick Cannon apologized late Wednesday “hurtful...      1\n",
      "7555  Kellyanne Conway says Trump resume coronavirus...      2\n",
      "7109  Cow burps fossil fuels: Methane emissions soar...      2\n",
      "\n",
      "[380 rows x 2 columns], 8:                                                Sentence  polit\n",
      "3506  Rather take hostile position confronting famil...      1\n",
      "3579  Alicia Silverstone recently shared special mem...      1\n",
      "1643  Unfortunately, website currently unavailable E...      1\n",
      "5608  Demonstrators gather front Texas State Capital...      2\n",
      "7056  The family tech CEO found dismembered Manhatta...      2\n",
      "...                                                 ...    ...\n",
      "8191  Washington — As school districts weigh whether...      2\n",
      "6049  New Ford Bronco debuts, challenges Jeep king h...      2\n",
      "8188  The Arizona Wildcats added transfers offseason...      2\n",
      "8551  Chinese researchers Tencent Security Labs foun...      2\n",
      "536   NASA’s Parker Solar Probe set close flyby Venu...      1\n",
      "\n",
      "[380 rows x 2 columns], 9:                                                Sentence  polit\n",
      "9373  London — Rebecca Wrixon knew working nanny pai...      2\n",
      "31    A former Miss Kentucky admitted exchanging sex...      1\n",
      "6353  Drew McIntyre scheduled defend WWE Championshi...      2\n",
      "8641  A handful upcoming titles Nintendo Switch's th...      2\n",
      "8616  Grocery chain Winn-Dixie require customers wea...      2\n",
      "...                                                 ...    ...\n",
      "173   The Los Angeles County coroner's office determ...      1\n",
      "4712  After signing reported -year, $ million extens...      1\n",
      "1146  Google announced Monday second-generation Pixe...      1\n",
      "7391  Following discovery private Super Mario Twitte...      2\n",
      "6012  Big crowds lack social distancing Jersey Shore...      2\n",
      "\n",
      "[380 rows x 2 columns], 10:                                                Sentence  polit\n",
      "2837  Kendall Jenner's hiking cover might've blown D...      1\n",
      "292   Sarah Tew/CNET It's nearly impossible find Nin...      1\n",
      "6585  Warning! The following contains spoilers Ameri...      2\n",
      "7485  In ways, stock market crash earlier year seems...      2\n",
      "1088  Jada Pinkett Smith | Instagram Everyone talkin...      1\n",
      "...                                                 ...    ...\n",
      "1560  COMMENTARY While Cam Newton odds-on favorite s...      1\n",
      "2981  Cases Covid- surging across United States’ sun...      1\n",
      "823   It’s finally happened. After months resisting ...      1\n",
      "5183  ⚡️ Read full article Motorious Deemed 'Ultimat...      2\n",
      "4381  Media playback unsupported device Media captio...      1\n",
      "\n",
      "[380 rows x 2 columns], 11:                                                Sentence  polit\n",
      "6732  On day Peters Township Public Library McMurray...      2\n",
      "3768  While hands-down best noise-cancelling headpho...      1\n",
      "5910  Sonoma County restaurants, wineries bars longe...      2\n",
      "9214  A Yelp study finds % business closures closed ...      2\n",
      "2902  Bray Wyatt officially reverted back Eater Worl...      1\n",
      "...                                                 ...    ...\n",
      "4073  Malika Andrews shows NBA courts look like insi...      1\n",
      "8657  Work home: More companies letting new hires wo...      2\n",
      "8768  WASHINGTON — Dr. Jerome Adams faces two diffic...      2\n",
      "6950  Are still collecting? [Update: Pre-orders live...      2\n",
      "6795  New York (CNN Business) Wells Fargo gone one A...      2\n",
      "\n",
      "[380 rows x 2 columns], 12:                                                Sentence  polit\n",
      "1732  © Nintendo Life Yesterday, probably already kn...      1\n",
      "7112  The Supreme Court cleared way second federal e...      2\n",
      "9464  Good evening Bucknuts subscribers! There plent...      2\n",
      "8837  “DHEC issued Public Health Order supporting tr...      2\n",
      "3339  There many standout moments Fox News Sunday an...      1\n",
      "...                                                 ...    ...\n",
      "4429  Google announced rolling mortgage information ...      1\n",
      "1615  Apps created Macs typically receive much cover...      1\n",
      "4769  “This issue one incident,” declared Rep. Alexa...      1\n",
      "9404  Refinery Once stay-at-home orders went place N...      2\n",
      "3611  Sonja Morgan’s new look achieved going knife. ...      1\n",
      "\n",
      "[380 rows x 2 columns], 13:                                                Sentence  polit\n",
      "6409  By – vote, Supreme Court allowed Trump adminis...      2\n",
      "1271  The Independent employs reporters around world...      1\n",
      "1259  Did know save preferences across digital devic...      1\n",
      "3075  WASHINGTON (AP) — Joe Biden said Friday night ...      1\n",
      "790   School buses parked Arlington County Bus Depot...      1\n",
      "...                                                 ...    ...\n",
      "7118  New York (CNN Business) Dr Disrespect, one pop...      2\n",
      "3692  The sudden spike popularity video conferencing...      1\n",
      "2714  The final Pokémon Go Fest weekly challenge run...      1\n",
      "1701  At foot certain shrubs Mojave Desert, fluffy w...      1\n",
      "7860  AppleInsider affiliate partnerships may earn c...      2\n",
      "\n",
      "[380 rows x 2 columns], 14:                                                Sentence  polit\n",
      "2742  One classic comedy came back much stronger aud...      1\n",
      "8064  (CNN) After Cleo -year-old Labrador went missi...      2\n",
      "6251  Products featured independently selected edito...      2\n",
      "288   Photo : OpturaDesign ( Shutterstock ) By desig...      1\n",
      "1093  WELLINGTON, New Zealand — The Australian white...      1\n",
      "...                                                 ...    ...\n",
      "6625  Superstrata What's better D printing bikes acc...      2\n",
      "5911  SALINE, Mich. (WXYZ) — The Washtenaw County He...      2\n",
      "8568  RIVERSIDE COUNTY, CA — The number people hospi...      2\n",
      "399   Photo : Jeep The -liter EcoDiesel V  Jeep Wran...      1\n",
      "658   The video start  Cancel Get daily royal round-...      1\n",
      "\n",
      "[380 rows x 2 columns], 15:                                                Sentence  polit\n",
      "6593  Flathead County reported  new positive cases C...      2\n",
      "1064  As Jada Pinkett Smith Will Smith's explosive \"...      1\n",
      "1339  This week NBCUniversal’s Peacock streaming ser...      1\n",
      "7124  As EA Sports rolls player ratings Madden NFL  ...      2\n",
      "5809  Houston leaders calling another lockdown numbe...      2\n",
      "...                                                 ...    ...\n",
      "5369  Having iPhone battery drain  percent  hours mi...      2\n",
      "2308  Blind auditions based appealing premise pure m...      1\n",
      "5138  The Pennsylvania Department Health reported , ...      2\n",
      "1879  This visualization shows moon formed collision...      1\n",
      "8797  (CNN) If people washed hands regularly, wore m...      2\n",
      "\n",
      "[380 rows x 2 columns], 16:                                                Sentence  polit\n",
      "1652  Apple transitioning line Macs Intel-based proc...      1\n",
      "6528  ANNOUNCING: We suing @Uber @Lyft misclassifyin...      2\n",
      "8153  Exclusive Kanye West's off-the-rails campaign ...      2\n",
      "2663  A -year-old California man arrested Las Vegas ...      1\n",
      "1483  Until two weeks ago, Dario Calmese didn’t know...      1\n",
      "...                                                 ...    ...\n",
      "1781  In searching Lakers’ various social media chan...      1\n",
      "246   Artist’s conception comet getting destroyed bl...      1\n",
      "1158  Life inside bubble NBA players: Golf, fishing ...      1\n",
      "4789  Twitter \"Barbara Walters done Trump's BS \" It ...      1\n",
      "2830  (CNN) — It’s get tough buy lumber sump pump wi...      1\n",
      "\n",
      "[380 rows x 2 columns], 17:                                                Sentence  polit\n",
      "6952  Star Wars: Squadrons let fans galaxy far, far ...      2\n",
      "7357  Curium crystals orange glow, researchers used ...      2\n",
      "4364  Toys Collectibles Action figures, statues, exc...      1\n",
      "6737  Central Michigan athletic director Mike Alford...      2\n",
      "2799  On July , , Associated Press reported Pentagon...      1\n",
      "...                                                 ...    ...\n",
      "493   Forget moment sheer cruelty stripping internat...      1\n",
      "5162  A couple enjoying swim n Richmond, Kentucky, s...      2\n",
      "7105  BERLIN — The first general election North Mace...      2\n",
      "9375  Some health authorities U.S. started narrowing...      2\n",
      "5543  Syracuse, N.Y. -- Sideline Cancer eliminated B...      2\n",
      "\n",
      "[380 rows x 2 columns], 18:                                                Sentence  polit\n",
      "5062  Washington DC (CNN Business) Efforts clean hou...      2\n",
      "9298  Ten people attended party Cape Cod town Chatha...      2\n",
      "815   ✖ One Sony's talking points PS fancy Solid Sta...      1\n",
      "1261  It’s officially ranking season, means it’s tim...      1\n",
      "1352  Amazon launching shopping carts track items sh...      1\n",
      "...                                                 ...    ...\n",
      "6497  The NFL NFL Players Association continue negot...      2\n",
      "2054  Padma Lakshmi revealed eats anywhere , , calor...      1\n",
      "1995  Two months ago, Israeli Prime Minister Benjami...      1\n",
      "8707  According ESPN, one biggest advantages Ravens ...      2\n",
      "3400  In month, fall school year begin many American...      1\n",
      "\n",
      "[380 rows x 2 columns], 19:                                                Sentence  polit\n",
      "4522  CLICK HERE you’re problem viewing photos mobil...      1\n",
      "5247  The story tortoise hare ranks one beloved Aeso...      2\n",
      "3510  Action movie fans, rejoice: Kevin Hart save bl...      1\n",
      "6553  Bari Weiss, high-profile editor writer New Yor...      2\n",
      "2087  In essay tucked back The Old Guard: Force Mult...      1\n",
      "...                                                 ...    ...\n",
      "466   I inadvertently took month-long break Animal C...      1\n",
      "6661  The Tompkins County Health Department says ten...      2\n",
      "975   If one available you, (k) easily become corner...      1\n",
      "4534  Staff Ellen DeGeneres' TV show reportedly deli...      1\n",
      "8462  We use range cookies give best possible browsi...      2\n",
      "\n",
      "[380 rows x 2 columns], 20:                                                Sentence  polit\n",
      "9108  Top- prospect Malaki Branham committed Ohio St...      2\n",
      "3116  Nick Cannon seemed contemplated suicide ... ca...      1\n",
      "2669  Facebook serious new spruced section. So serio...      1\n",
      "1826  Xiaomi launched several new products event tod...      1\n",
      "6265  Scientists warned last year’s Ridgecrest quake...      2\n",
      "...                                                 ...    ...\n",
      "3320  (CNN) Finally, good news  For first time since...      1\n",
      "9204  The measure response say Trump's effort ban Mu...      2\n",
      "5773  Armie Hammer Elizabeth Chambers recently becam...      2\n",
      "730   Although All-Stars chosen reality competition ...      1\n",
      "145   San Francisco (CNN Business) For decades, Sili...      1\n",
      "\n",
      "[380 rows x 2 columns], 21:                                                Sentence  polit\n",
      "6819  Los Angeles County confirmed cases coronavirus...      2\n",
      "9421  'Symbols bigotry racism': House votes remove C...      2\n",
      "2063  Photo: Joe Murphy/NBAE via Getty Images To quo...      1\n",
      "8804  EMBED >More News Videos Gov. Roy Cooper gives ...      2\n",
      "7933  Mercedes’ Lewis Hamilton enjoyed peerless run ...      2\n",
      "...                                                 ...    ...\n",
      "4477  Get breaking news alerts special reports. The ...      1\n",
      "8889  Recently, Japanese magazine Famitsu hosted pop...      2\n",
      "1769  Discussion whether markets top lasted one day,...      1\n",
      "972   Three Arizona summer-school teachers followed ...      1\n",
      "7936  Umpire Richard Illingworth (right) watches fel...      2\n",
      "\n",
      "[380 rows x 2 columns], 22:                                                Sentence  polit\n",
      "4062  Larsa Pippen -year-old daughter Sophia looking...      1\n",
      "7225  Uniform mask-wearing one hair salon Missouri m...      2\n",
      "5164  Coronavirus infectious disease confirmed  mill...      2\n",
      "5232  In Arizona, Ms. Wysong, , said willing return ...      2\n",
      "3757  California Gov. Gavin Newsom denied Monday Whi...      1\n",
      "...                                                 ...    ...\n",
      "4754  Let's get qualifiers way first. Yes, one game....      1\n",
      "3647  Insomniac Games announced Spider-Man: Miles Mo...      1\n",
      "7413  (CNN) Covid- cause wide variety symptoms , new...      2\n",
      "8289  CONNECTICUT —There four new resident deaths on...      2\n",
      "6994  The New England Patriots Wednesday,  pm reach ...      2\n",
      "\n",
      "[380 rows x 2 columns], 23:                                                Sentence  polit\n",
      "4580  Four months long time. So we’re getting reacqu...      1\n",
      "387   Eminem slams Drew Brees, people refuse wear ma...      1\n",
      "1635  “How open visitors get everyone back work prot...      1\n",
      "4219  Health experts heat waves become hazardous lon...      1\n",
      "5092  University Vermont post-doctoral researcher Mo...      2\n",
      "...                                                 ...    ...\n",
      "8216  President Donald Trump refused commit acceptin...      2\n",
      "3846  Your mobile devices, including iPhone Android ...      1\n",
      "273   Gordon Chang, author \"The Coming Collapse Chin...      1\n",
      "896   CBS All Access today released first trailer St...      1\n",
      "7649  'They need exposed': D.J. Swearinger calls for...      2\n",
      "\n",
      "[380 rows x 2 columns], 24:                                                Sentence  polit\n",
      "1078  TikTok The -year-old actress ready Deacon Phil...      1\n",
      "4129  In The Know Noted heartthrob Harry Styles cycl...      1\n",
      "6253  Tiger Woods return PGA Tour week first time fi...      2\n",
      "5538  Justin Thomas model efficiency consistency thr...      2\n",
      "5860  (CNN) President Donald Trump Saturday finally ...      2\n",
      "...                                                 ...    ...\n",
      "2948  Fear City: New York vs The Mafia A new Netflix...      1\n",
      "4714  Let's get qualifiers way first. Yes, one game....      1\n",
      "6853  The Centers Disease Control added multiple new...      2\n",
      "1841  The Astros placed reliever Joe Smith restricte...      1\n",
      "3638  National fast food chain Taco Bell says remove...      1\n",
      "\n",
      "[377 rows x 2 columns], 25: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 26: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 27: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 28: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 29: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 30: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 31: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 32: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 33: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 34: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 35: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 36: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 37: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 38: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 39: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 40: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 41: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 42: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 43: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 44: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 45: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 46: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 47: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 48: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 49: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 50: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 51: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 52: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 53: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 54: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 55: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 56: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 57: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 58: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 59: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 60: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 61: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 62: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 63: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 64: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 65: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 66: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 67: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 68: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 69: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 70: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 71: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 72: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 73: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 74: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 75: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 76: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 77: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 78: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 79: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 80: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 81: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 82: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 83: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 84: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 85: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 86: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 87: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 88: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 89: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 90: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 91: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 92: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 93: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 94: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 95: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 96: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 97: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 98: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 99: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 100: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 101: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 102: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 103: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 104: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 105: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 106: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 107: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 108: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 109: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 110: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 111: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 112: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 113: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 114: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 115: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 116: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 117: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 118: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 119: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 120: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 121: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 122: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 123: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 124: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 125: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 126: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 127: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 128: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 129: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 130: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 131: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 132: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 133: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 134: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 135: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 136: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 137: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 138: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 139: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 140: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 141: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 142: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 143: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 144: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 145: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 146: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 147: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 148: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 149: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 150: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 151: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 152: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 153: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 154: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 155: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 156: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 157: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 158: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 159: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 160: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 161: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 162: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 163: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 164: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 165: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 166: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 167: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 168: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 169: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 170: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 171: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 172: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 173: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 174: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 175: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 176: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 177: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 178: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 179: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 180: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 181: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 182: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 183: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 184: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 185: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 186: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 187: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 188: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 189: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 190: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 191: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 192: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 193: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 194: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 195: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 196: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 197: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 198: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 199: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 200: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 201: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 202: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 203: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 204: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 205: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 206: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 207: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 208: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 209: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 210: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 211: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 212: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 213: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 214: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 215: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 216: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 217: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 218: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 219: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 220: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 221: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 222: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 223: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 224: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 225: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 226: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 227: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 228: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 229: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 230: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 231: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 232: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 233: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 234: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 235: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 236: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 237: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 238: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 239: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 240: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 241: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 242: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 243: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 244: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 245: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 246: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 247: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 248: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 249: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 250: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 251: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 252: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 253: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 254: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 255: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 256: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 257: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 258: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 259: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 260: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 261: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 262: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 263: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 264: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 265: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 266: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 267: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 268: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 269: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 270: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 271: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 272: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 273: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 274: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 275: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 276: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 277: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 278: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 279: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 280: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 281: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 282: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 283: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 284: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 285: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 286: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 287: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 288: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 289: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 290: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 291: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 292: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 293: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 294: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 295: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 296: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 297: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 298: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 299: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 300: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 301: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 302: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 303: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 304: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 305: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 306: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 307: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 308: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 309: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 310: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 311: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 312: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 313: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 314: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 315: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 316: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 317: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 318: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 319: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 320: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 321: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 322: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 323: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 324: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 325: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 326: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 327: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 328: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 329: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 330: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 331: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 332: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 333: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 334: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 335: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 336: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 337: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 338: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 339: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 340: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 341: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 342: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 343: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 344: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 345: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 346: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 347: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 348: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 349: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 350: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 351: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 352: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 353: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 354: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 355: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 356: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 357: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 358: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 359: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 360: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 361: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 362: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 363: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 364: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 365: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 366: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 367: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 368: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 369: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 370: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 371: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 372: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 373: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 374: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 375: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 376: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 377: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 378: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: [], 379: Empty DataFrame\n",
      "Columns: [Sentence, polit]\n",
      "Index: []}\n"
     ]
    }
   ],
   "source": [
    "myclient = pymongo.MongoClient(\"mongodb://localhost:27017/\")\n",
    "mydb = myclient[\"news\"]\n",
    "mycol = mydb[\"political\"]\n",
    "left_query = { \"nutrition.political_bias.subfeatures.value\": 0}\n",
    "right_query = { \"nutrition.political_bias.subfeatures.value\": 4}\n",
    "\n",
    "\n",
    "lr = mycol.find(left_query).limit(4800)\n",
    "rr = mycol.find(right_query).limit(4697)\n",
    "left_data = pd.DataFrame(columns=['Sentence'],data=[i['article']['text'] for i in list(lr)] )\n",
    "right_data = pd.DataFrame(columns=['Sentence'],data=[i['article']['text'] for i in list(rr)] )\n",
    "left_data['polit'] = 1\n",
    "right_data['polit'] = 2\n",
    "pc = r\"\\d+\\.?\\d*\"\n",
    "\n",
    "stop = stopwords.words('english')\n",
    "b_set = pd.concat( [left_data,right_data], axis=0,ignore_index=True)\n",
    "b_set['Sentence'] = b_set['Sentence'].apply(lambda w:' '.join([re.sub(pc,\"\",word) for word in str(w).split() if word not in (stop)]))\n",
    "b_set = b_set.sample(frac = 1)\n",
    "sub_df = {}\n",
    "rand_ind = np.random.permutation(9497)\n",
    "sp = 380\n",
    "for i in range(sp):\n",
    "    indices = rand_ind[sp*i : sp*i + sp]\n",
    "    sub_df[i] = b_set.iloc[indices]\n",
    "print(b_set)\n",
    "print(sub_df)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Sentence</th>\n",
       "      <th>polit</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2978</th>\n",
       "      <td>The beach question along Stanley Lake despite ...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4129</th>\n",
       "      <td>Surprise! Kate Middleton switched look summer,...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2624</th>\n",
       "      <td>Coral colony multiple stages bleaching mortali...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4803</th>\n",
       "      <td>Secretary State Mike Pompeo announced Wednesda...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2840</th>\n",
       "      <td>Support high school sports coverage becoming d...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3462</th>\n",
       "      <td>It moment make Mattia Binotto wince. As seeing...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4521</th>\n",
       "      <td>The Dow Jones Industrial Average rose fourth s...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3997</th>\n",
       "      <td>(CNN) A suspect fatal shooting police officer ...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3182</th>\n",
       "      <td>'Quarantine ovah': Oprah Winfrey Gayle King sh...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4737</th>\n",
       "      <td>Thousands companies face restrictions storing ...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4623</th>\n",
       "      <td>Michigan rival college coaches join Gov. Whitm...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1855</th>\n",
       "      <td>The Paycheck Protection Program, PPP, criticiz...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1099</th>\n",
       "      <td>Google's newest ad ban classic \"better late ne...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4829</th>\n",
       "      <td>A common drug, already approved Food Drug Admi...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>477</th>\n",
       "      <td>What already spectacular event made even intri...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>685</th>\n",
       "      <td>While average amount storage high-end smartpho...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>366</th>\n",
       "      <td>Naughty Dog released The Last Us Part II June ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>Bitcoin’s (BTC) active supply hit -month low o...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3630</th>\n",
       "      <td>RELATED STORIES &amp; VIDEOS: SAN JOSE, Calif. (KG...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4660</th>\n",
       "      <td>Ultimate Fighting Championship (UFC) Featherwe...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4729</th>\n",
       "      <td>Prosecutors say Maxwell, pleaded guilty involv...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1248</th>\n",
       "      <td>Unfortunately, website currently unavailable E...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4327</th>\n",
       "      <td>Sheldon Riley waste voice trifling matters. Li...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4509</th>\n",
       "      <td>Gene J. Puskar/Associated Press In , Baltimo...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1202</th>\n",
       "      <td>Image copyright Reuters Canadian Prime Ministe...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3581</th>\n",
       "      <td>U.S. Navy officials Monday working determine c...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2926</th>\n",
       "      <td>The LHCb collaboration CERN announced discover...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>856</th>\n",
       "      <td>Former Food Drug Administration commissioner S...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3340</th>\n",
       "      <td>Global oil demand rebounded somewhat faster pr...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>137</th>\n",
       "      <td>Welcome The Hill’s Campaign Report, daily rund...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>474</th>\n",
       "      <td>The Workday Charity Open turned ball-striking ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3168</th>\n",
       "      <td>I love books films take place really short per...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1245</th>\n",
       "      <td>President Donald Trump falsely claimed golfs l...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>735</th>\n",
       "      <td>NASA sending Curiosity rover journey new area ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524</th>\n",
       "      <td>Even though Pixel a didn’t pan way Google want...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2732</th>\n",
       "      <td>Los Alamos research reveals wildfire smoke plu...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2349</th>\n",
       "      <td>Screenshot : Rockstar Games/Emojipedia ( Fair ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2934</th>\n",
       "      <td>The CEO Goya Foods stood decision compliment P...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1959</th>\n",
       "      <td>Unfortunately, website currently unavailable E...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3386</th>\n",
       "      <td>NASCAR back -mile circuit Sunday’s Quaker Stat...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>186</th>\n",
       "      <td>There’s rumor going around week new, -core Thr...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4170</th>\n",
       "      <td>Carolina Panthers running back Christian McCaf...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2091</th>\n",
       "      <td>The Jacksonville Jaguars parried offers Yannic...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>637</th>\n",
       "      <td>The Navy trolling Army Tom Hanks' new WWII fil...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1667</th>\n",
       "      <td>The bottom Eastern Conference playoff picture ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2255</th>\n",
       "      <td>Advertisement Friends said Fahim Saleh (left r...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4148</th>\n",
       "      <td>(CNN) A former nursing assistant Veterans Affa...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3687</th>\n",
       "      <td>Epidemiologist Arnaud Fontanet, member team co...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3443</th>\n",
       "      <td>The former Florida insurance agent fired exple...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2950</th>\n",
       "      <td>President Trump said direct Treasury Departmen...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               Sentence  polit\n",
       "2978  The beach question along Stanley Lake despite ...      2\n",
       "4129  Surprise! Kate Middleton switched look summer,...      2\n",
       "2624  Coral colony multiple stages bleaching mortali...      2\n",
       "4803  Secretary State Mike Pompeo announced Wednesda...      2\n",
       "2840  Support high school sports coverage becoming d...      2\n",
       "3462  It moment make Mattia Binotto wince. As seeing...      2\n",
       "4521  The Dow Jones Industrial Average rose fourth s...      2\n",
       "3997  (CNN) A suspect fatal shooting police officer ...      2\n",
       "3182  'Quarantine ovah': Oprah Winfrey Gayle King sh...      2\n",
       "4737  Thousands companies face restrictions storing ...      2\n",
       "4623  Michigan rival college coaches join Gov. Whitm...      2\n",
       "1855  The Paycheck Protection Program, PPP, criticiz...      1\n",
       "1099  Google's newest ad ban classic \"better late ne...      1\n",
       "4829  A common drug, already approved Food Drug Admi...      2\n",
       "477   What already spectacular event made even intri...      1\n",
       "685   While average amount storage high-end smartpho...      1\n",
       "366   Naughty Dog released The Last Us Part II June ...      1\n",
       "37    Bitcoin’s (BTC) active supply hit -month low o...      1\n",
       "3630  RELATED STORIES & VIDEOS: SAN JOSE, Calif. (KG...      2\n",
       "4660  Ultimate Fighting Championship (UFC) Featherwe...      2\n",
       "4729  Prosecutors say Maxwell, pleaded guilty involv...      2\n",
       "1248  Unfortunately, website currently unavailable E...      1\n",
       "4327  Sheldon Riley waste voice trifling matters. Li...      2\n",
       "4509    Gene J. Puskar/Associated Press In , Baltimo...      2\n",
       "1202  Image copyright Reuters Canadian Prime Ministe...      1\n",
       "3581  U.S. Navy officials Monday working determine c...      2\n",
       "2926  The LHCb collaboration CERN announced discover...      2\n",
       "856   Former Food Drug Administration commissioner S...      1\n",
       "3340  Global oil demand rebounded somewhat faster pr...      2\n",
       "137   Welcome The Hill’s Campaign Report, daily rund...      1\n",
       "474   The Workday Charity Open turned ball-striking ...      1\n",
       "3168  I love books films take place really short per...      2\n",
       "1245  President Donald Trump falsely claimed golfs l...      1\n",
       "735   NASA sending Curiosity rover journey new area ...      1\n",
       "524   Even though Pixel a didn’t pan way Google want...      1\n",
       "2732  Los Alamos research reveals wildfire smoke plu...      2\n",
       "2349  Screenshot : Rockstar Games/Emojipedia ( Fair ...      1\n",
       "2934  The CEO Goya Foods stood decision compliment P...      2\n",
       "1959  Unfortunately, website currently unavailable E...      1\n",
       "3386  NASCAR back -mile circuit Sunday’s Quaker Stat...      2\n",
       "186   There’s rumor going around week new, -core Thr...      1\n",
       "4170  Carolina Panthers running back Christian McCaf...      2\n",
       "2091  The Jacksonville Jaguars parried offers Yannic...      1\n",
       "637   The Navy trolling Army Tom Hanks' new WWII fil...      1\n",
       "1667  The bottom Eastern Conference playoff picture ...      1\n",
       "2255  Advertisement Friends said Fahim Saleh (left r...      1\n",
       "4148  (CNN) A former nursing assistant Veterans Affa...      2\n",
       "3687  Epidemiologist Arnaud Fontanet, member team co...      2\n",
       "3443  The former Florida insurance agent fired exple...      2\n",
       "2950  President Trump said direct Treasury Departmen...      2"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub_df[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0 0 0 ... 0 0 0]\n",
      " [0 0 0 ... 0 0 0]\n",
      " [0 0 0 ... 0 0 0]\n",
      " ...\n",
      " [0 0 0 ... 0 0 0]\n",
      " [0 0 0 ... 0 0 0]\n",
      " [0 0 0 ... 0 0 0]]\n",
      "[[0 0 0 ... 0 0 0]\n",
      " [0 0 0 ... 0 0 0]\n",
      " [0 0 0 ... 0 0 0]\n",
      " ...\n",
      " [0 0 0 ... 0 0 0]\n",
      " [0 0 0 ... 0 0 0]\n",
      " [0 0 0 ... 0 0 0]]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>___</th>\n",
       "      <th>aa</th>\n",
       "      <th>aaa</th>\n",
       "      <th>aapl</th>\n",
       "      <th>aaron</th>\n",
       "      <th>aaron judge</th>\n",
       "      <th>aaron rodgers</th>\n",
       "      <th>ab</th>\n",
       "      <th>abandon</th>\n",
       "      <th>abandoned</th>\n",
       "      <th>...</th>\n",
       "      <th>zoom home</th>\n",
       "      <th>zoos</th>\n",
       "      <th>zuckerberg</th>\n",
       "      <th>zuckerberg said</th>\n",
       "      <th>zurich</th>\n",
       "      <th>zv</th>\n",
       "      <th>zymet</th>\n",
       "      <th>µg</th>\n",
       "      <th>μg</th>\n",
       "      <th>μm</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 30000 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   ___  aa  aaa  aapl  aaron  aaron judge  aaron rodgers  ab  abandon  \\\n",
       "0    0   0    0     0      0            0              0   0        0   \n",
       "1    0   0    0     0      0            0              0   0        0   \n",
       "2    0   0    0     0      0            0              0   0        0   \n",
       "3    0   0    0     0      0            0              0   0        0   \n",
       "4    0   0    0     0      0            0              0   0        0   \n",
       "\n",
       "   abandoned  ...  zoom home  zoos  zuckerberg  zuckerberg said  zurich  zv  \\\n",
       "0          0  ...          0     0           0                0       0   0   \n",
       "1          0  ...          0     0           0                0       0   0   \n",
       "2          0  ...          0     0           0                0       0   0   \n",
       "3          0  ...          0     0           0                0       0   0   \n",
       "4          0  ...          0     0           0                0       0   0   \n",
       "\n",
       "   zymet  µg  μg  μm  \n",
       "0      0   0   0   0  \n",
       "1      0   0   0   0  \n",
       "2      0   0   0   0  \n",
       "3      0   0   0   0  \n",
       "4      0   0   0   0  \n",
       "\n",
       "[5 rows x 30000 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import joblib\n",
    "\n",
    "# tokenize document \n",
    "tv=TfidfVectorizer(use_idf=True,smooth_idf=True,encoding='utf-8',ngram_range=(1,2),max_features=30000)\n",
    "fit_tv=tv.fit_transform(b_set['Sentence']).astype('int').toarray()\n",
    "#print(tv.get_feature_names())\n",
    "#print(fit_tv.shape)\n",
    "#print(fit_tv)\n",
    "term_matrix = pd.DataFrame(fit_tv, columns=tv.get_feature_names())\n",
    "\n",
    "# split the train/test dataset \n",
    "X_train,X_test,Y_train,Y_test=train_test_split(fit_tv, b_set['polit'].astype('int').values\n",
    "                                               ,test_size=0.1,random_state = None)\n",
    "joblib.dump(X_test, '30000X.test')\n",
    "joblib.dump(Y_test, '30000Y.test')\n",
    "# X_test = joblib.load('100000X.test')\n",
    "# Y_test = joblib.load('100000Y.test')\n",
    "\n",
    "print(X_test)\n",
    "print(X_train)\n",
    "term_matrix.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.511578947368421\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "lr_clf = LogisticRegression()\n",
    "lr_clf.fit(X_train, Y_train)\n",
    "sc = lr_clf.score(X_test, Y_test)\n",
    "print(sc)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4950, 30000)\n",
      "[2 2 1 ... 2 2 2]\n"
     ]
    }
   ],
   "source": [
    "X_train\n",
    "Y_train\n",
    "print(X_train.shape)\n",
    "print(Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/anaconda3/lib/python3.8/site-packages/sklearn/naive_bayes.py:455: RuntimeWarning: divide by zero encountered in log\n",
      "  n_ij = - 0.5 * np.sum(np.log(2. * np.pi * self.sigma_[i, :]))\n",
      "/usr/local/anaconda3/lib/python3.8/site-packages/sklearn/naive_bayes.py:456: RuntimeWarning: invalid value encountered in true_divide\n",
      "  n_ij -= 0.5 * np.sum(((X - self.theta_[i, :]) ** 2) /\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gnb: 0.511579\n",
      "mb: 0.511579\n",
      "cb: 0.511579\n",
      "bb: 0.511579\n"
     ]
    }
   ],
   "source": [
    "from sklearn.datasets import load_iris\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.naive_bayes import ComplementNB\n",
    "from sklearn.naive_bayes import BernoulliNB \n",
    "from sklearn.naive_bayes import CategoricalNB\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "training_accuracy = []\n",
    "test_accuracy = []\n",
    "test_accuracy2 = []\n",
    "test_accuracy3 = []\n",
    "test_accuracy4 = []\n",
    "gnb = GaussianNB()\n",
    "mb = MultinomialNB()\n",
    "cb = ComplementNB()\n",
    "bb = BernoulliNB()\n",
    "# sets = [0+(i/100)*2 for i in range(1,10)]\n",
    "# for i in sets:\n",
    "#     X_train,X_test,Y_train,Y_test=train_test_split(fit_tv, b_set['polit'].astype('int').values\n",
    "#                                                ,test_size=i,random_state = 7)\n",
    "#     y_pred = gnb.fit(X_train, Y_train).predict(X_test)\n",
    "#     y_pred2 = mb.fit(X_train, Y_train).predict(X_test)\n",
    "#     y_pred3 = cb.fit(X_train, Y_train).predict(X_test)\n",
    "#     y_pred4 = bb.fit(X_train, Y_train).predict(X_test)\n",
    "#     test_accuracy.append((Y_test == y_pred).sum()/X_test.shape[0])\n",
    "#     test_accuracy2.append((Y_test == y_pred2).sum()/X_test.shape[0])\n",
    "#     test_accuracy3.append((Y_test == y_pred3).sum()/X_test.shape[0])\n",
    "#     test_accuracy4.append((Y_test == y_pred4).sum()/X_test.shape[0])\n",
    "# # plt.plot(neighbors_setting, training_accuracy, label=\"training accuracy\")\n",
    "# plt.plot(sets, test_accuracy, label=\"GaussianNB accuracy\")\n",
    "# plt.plot(sets, test_accuracy2, label=\"MultinomialNB accuracy\")\n",
    "# plt.plot(sets, test_accuracy3, label=\"ComplementNB accuracy\")\n",
    "# plt.plot(sets, test_accuracy4, label=\"BernoulliNB accuracy\")\n",
    "# plt.ylabel(\"Accuracy\")\n",
    "# plt.xlabel(\"test:train\")\n",
    "# plt.legend()\n",
    "# plt.show()\n",
    "\n",
    "\n",
    "cgb = CategoricalNB()\n",
    "y_pred = gnb.fit(X_train, Y_train).predict(X_test)\n",
    "y_pred2 = mb.fit(X_train, Y_train).predict(X_test)\n",
    "y_pred3 = cb.fit(X_train, Y_train).predict(X_test)\n",
    "y_pred4 = bb.fit(X_train, Y_train).predict(X_test)\n",
    "print(\"gnb: %f\" % ((Y_test == y_pred).sum()/X_test.shape[0]))\n",
    "print(\"mb: %f\" % ((Y_test == y_pred2).sum()/X_test.shape[0]))\n",
    "print(\"cb: %f\" % ((Y_test == y_pred3).sum()/X_test.shape[0]))\n",
    "print(\"bb: %f\" % ((Y_test == y_pred4).sum()/X_test.shape[0]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 bb: 0.483158\n",
      "2 bb: 0.483158\n",
      "3 bb: 0.483158\n",
      "4 bb: 0.483158\n",
      "5 bb: 0.483158\n",
      "6 bb: 0.483158\n",
      "7 bb: 0.483158\n",
      "8 bb: 0.483158\n",
      "9 bb: 0.483158\n",
      "10 bb: 0.483158\n",
      "11 bb: 0.483158\n",
      "12 bb: 0.483158\n",
      "13 bb: 0.483158\n",
      "14 bb: 0.483158\n",
      "15 bb: 0.483158\n",
      "16 bb: 0.483158\n",
      "17 bb: 0.483158\n",
      "18 bb: 0.483158\n",
      "19 bb: 0.483158\n",
      "20 bb: 0.483158\n",
      "21 bb: 0.483158\n",
      "22 bb: 0.483158\n",
      "23 bb: 0.483158\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "Found array with 0 sample(s) (shape=(0, 30000)) while a minimum of 1 is required.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-12-659920b3e234>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      9\u001b[0m     \u001b[0mX_test\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mjoblib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'30000X.test'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m     \u001b[0mY_test\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mjoblib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'30000Y.test'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# use the local-saved test data\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 11\u001b[0;31m     \u001b[0mbb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpartial_fit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0msp\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0msp\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY_train\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0msp\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0msp\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclasses\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     12\u001b[0m     \u001b[0my_pred4\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"%d bb: %f\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mY_test\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0my_pred4\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0mX_test\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/anaconda3/lib/python3.8/site-packages/sklearn/naive_bayes.py\u001b[0m in \u001b[0;36mpartial_fit\u001b[0;34m(self, X, y, classes, sample_weight)\u001b[0m\n\u001b[1;32m    550\u001b[0m         \u001b[0mself\u001b[0m \u001b[0;34m:\u001b[0m \u001b[0mobject\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    551\u001b[0m         \"\"\"\n\u001b[0;32m--> 552\u001b[0;31m         \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_check_X_y\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    553\u001b[0m         \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_features\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    554\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/anaconda3/lib/python3.8/site-packages/sklearn/naive_bayes.py\u001b[0m in \u001b[0;36m_check_X_y\u001b[0;34m(self, X, y)\u001b[0m\n\u001b[1;32m    985\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    986\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_check_X_y\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 987\u001b[0;31m         \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_check_X_y\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    988\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbinarize\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    989\u001b[0m             \u001b[0mX\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbinarize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mthreshold\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbinarize\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/anaconda3/lib/python3.8/site-packages/sklearn/naive_bayes.py\u001b[0m in \u001b[0;36m_check_X_y\u001b[0;34m(self, X, y)\u001b[0m\n\u001b[1;32m    478\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    479\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_check_X_y\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 480\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_validate_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maccept_sparse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'csr'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    481\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    482\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_update_class_log_prior\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclass_prior\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/anaconda3/lib/python3.8/site-packages/sklearn/base.py\u001b[0m in \u001b[0;36m_validate_data\u001b[0;34m(self, X, y, reset, validate_separately, **check_params)\u001b[0m\n\u001b[1;32m    430\u001b[0m                 \u001b[0my\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcheck_array\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mcheck_y_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    431\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 432\u001b[0;31m                 \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcheck_X_y\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mcheck_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    433\u001b[0m             \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    434\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/anaconda3/lib/python3.8/site-packages/sklearn/utils/validation.py\u001b[0m in \u001b[0;36minner_f\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     70\u001b[0m                           FutureWarning)\n\u001b[1;32m     71\u001b[0m         \u001b[0mkwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0marg\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0marg\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 72\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     73\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0minner_f\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     74\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/anaconda3/lib/python3.8/site-packages/sklearn/utils/validation.py\u001b[0m in \u001b[0;36mcheck_X_y\u001b[0;34m(X, y, accept_sparse, accept_large_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, multi_output, ensure_min_samples, ensure_min_features, y_numeric, estimator)\u001b[0m\n\u001b[1;32m    793\u001b[0m         \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"y cannot be None\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    794\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 795\u001b[0;31m     X = check_array(X, accept_sparse=accept_sparse,\n\u001b[0m\u001b[1;32m    796\u001b[0m                     \u001b[0maccept_large_sparse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0maccept_large_sparse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    797\u001b[0m                     \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0morder\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0morder\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/anaconda3/lib/python3.8/site-packages/sklearn/utils/validation.py\u001b[0m in \u001b[0;36minner_f\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     70\u001b[0m                           FutureWarning)\n\u001b[1;32m     71\u001b[0m         \u001b[0mkwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0marg\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0marg\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 72\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     73\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0minner_f\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     74\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/anaconda3/lib/python3.8/site-packages/sklearn/utils/validation.py\u001b[0m in \u001b[0;36mcheck_array\u001b[0;34m(array, accept_sparse, accept_large_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, ensure_min_samples, ensure_min_features, estimator)\u001b[0m\n\u001b[1;32m    648\u001b[0m         \u001b[0mn_samples\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_num_samples\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    649\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mn_samples\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0mensure_min_samples\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 650\u001b[0;31m             raise ValueError(\"Found array with %d sample(s) (shape=%s) while a\"\n\u001b[0m\u001b[1;32m    651\u001b[0m                              \u001b[0;34m\" minimum of %d is required%s.\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    652\u001b[0m                              % (n_samples, array.shape, ensure_min_samples,\n",
      "\u001b[0;31mValueError\u001b[0m: Found array with 0 sample(s) (shape=(0, 30000)) while a minimum of 1 is required."
     ]
    }
   ],
   "source": [
    "from sklearn.datasets import load_iris\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.naive_bayes import BernoulliNB \n",
    "bb = BernoulliNB()\n",
    "for i in range(1,25):\n",
    "    X_train,X_test,Y_train,Y_test=train_test_split(fit_tv, b_set['polit'].astype('int').values\n",
    "                                               ,test_size=0.1,random_state = i)\n",
    "    # joblib.dump(X_test, '10000X.test')\n",
    "    X_test = joblib.load('30000X.test')\n",
    "    Y_test = joblib.load('30000Y.test') # use the local-saved test data\n",
    "    bb.partial_fit(X_train[sp*(i-1):sp*i], Y_train[sp*(i-1):sp*i], classes=np.array([1, 2]))\n",
    "    y_pred4 = bb.predict(X_test)\n",
    "    print(\"%d bb: %f\" % (i,(Y_test == y_pred4).sum()/X_test.shape[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 bb: 0.483158\n",
      "2 bb: 0.483158\n",
      "3 bb: 0.483158\n",
      "4 bb: 0.483158\n",
      "5 bb: 0.483158\n",
      "6 bb: 0.483158\n",
      "7 bb: 0.483158\n",
      "8 bb: 0.483158\n",
      "9 bb: 0.483158\n",
      "10 bb: 0.483158\n",
      "11 bb: 0.483158\n",
      "12 bb: 0.483158\n",
      "13 bb: 0.483158\n",
      "14 bb: 0.483158\n",
      "15 bb: 0.483158\n",
      "16 bb: 0.483158\n",
      "17 bb: 0.483158\n",
      "18 bb: 0.483158\n",
      "19 bb: 0.483158\n",
      "20 bb: 0.483158\n",
      "21 bb: 0.483158\n",
      "22 bb: 0.483158\n",
      "23 bb: 0.483158\n",
      "24 bb: 0.483158\n"
     ]
    }
   ],
   "source": [
    "from sklearn.datasets import load_iris\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.naive_bayes import BernoulliNB \n",
    "bb = BernoulliNB()\n",
    "for i in range(1,25):\n",
    "    X_train,X_test,Y_train,Y_test=train_test_split(fit_tv, b_set['polit'].astype('int').values\n",
    "                                               ,test_size=0.1,random_state = i)\n",
    "    # joblib.dump(X_test, '10000X.test')\n",
    "    X_test = joblib.load('30000X.test')\n",
    "    Y_test = joblib.load('30000Y.test') # use the local-saved test data\n",
    "    bb.fit(X_train, Y_train)\n",
    "    y_pred4 = bb.predict(X_test)\n",
    "    print(\"%d bb: %f\" % (i,(Y_test == y_pred4).sum()/X_test.shape[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.datasets import load_iris\n",
    "from sklearn.svm import LinearSVC, SVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LinearSVC()"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lsvc=LinearSVC()\n",
    "lsvc.fit(X_train,Y_train.astype('int'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.svm import SVR\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "import numpy as np\n",
    "regr = make_pipeline(StandardScaler(), SVR(C=1.0, epsilon=0.2))\n",
    "regr.fit(X_train, Y_train)\n",
    "print(regr.score(X_train, Y_train))\n",
    "print(regr.score(X_test, Y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# svm\n",
    "clf = SVC(C=0.8, kernel='linear', gamma=20, decision_function_shape='ovr')\n",
    "clf.fit(X_train, Y_train.ravel())\n",
    "print (clf.score(X_train, Y_train))\n",
    "print (clf.score(X_test, Y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2 0.511578947368421\n",
      "3 0.511578947368421\n",
      "4 0.511578947368421\n",
      "5 0.511578947368421\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.datasets import make_classification\n",
    "# X_train, Y_train = make_classification(n_samples=4000, n_features=30000,\n",
    "#                            n_informative=2, n_redundant=0,\n",
    "#                            random_state=0, shuffle=True)\n",
    "for i in range(2,10):\n",
    "    clf = RandomForestClassifier(max_depth=i, random_state=0)\n",
    "    clf.fit(X_train, Y_train)\n",
    "    print(i,clf.score(X_test,Y_test))    \n",
    "#print(clf.predict(X_test))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.965\n"
     ]
    }
   ],
   "source": [
    "print(clf.score(X_train,Y_train))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
